package com.chance.cc.crawler.development.bootstrap.dcdapp;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.internalDownload;

/**
 * @author lt
 * @version 1.0
 * @date 2021-05-10 14:21:46
 * @email okprog@sina.com
 * 懂车帝新闻、视频启动类
 */
public class DCDStart  {
    private static final String domain = "dcdapp";
    private static final String siteBiz = "news_realtime";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
//        proxy.setUsername("H7307T4706B25G4D");
//        proxy.setPassword("05B4877CC39192C0");
    }

    public static void main(String[] args) {
        //设置https协议访问
        System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");

        //视频采集
//        CrawlerRequestRecord videoCrawler = doVideoCrawler();
        //文章采集
        CrawlerRequestRecord doCommonCrawler = doCommonCrawler();
        //问答采集
        CrawlerRequestRecord seriesCrawler = doSeriesCrawler();

        CrawlerRequestRecord keywordRecord = CrawlerRequestRecord.builder()
                .startPageRequest("dcdapp_series_keyword",turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/"+domain+"/keys?site=series")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController crawlerController = DevCrawlerController.builder()
                .triggerInfo(domain, domain + "_trigger", System.currentTimeMillis(), domain + "_job")
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline("console") //控制台输t出
//                .fileResultPipeline("file","D:\\chance\\data\\dcdapp\\dcdapp_article.json", false)
//                .requestRecord(seriesCrawler)  //more job
                .requestRecord(doCommonCrawler)
//                .supportRecord(keywordRecord)
                .crawlerThreadNum(1)
                .build();
        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*2,null));
        doCommonCrawler.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        seriesCrawler.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        crawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        crawlerController.start();



    }


    private static CrawlerRequestRecord doSeriesCrawler() {
        String site = "Common_A_V";

        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);
        turnPageFilterInfo.setCurCount(0);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .httpUrl("https://www.dongchedi.com/")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*2,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .turnPageFilterInfo(turnPageFilterInfo)
                .needWashed(false)
                .needParsed(false)
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        return requestRecord;
    }

    private static CrawlerRequestRecord doCommonCrawler() {
        String site = "Common_A_V";

        FilterInfo turnPageFilterInfo = new FilterInfo();
        turnPageFilterInfo.setFilter(CrawlerEnum.CrawlerRecordFilter.count);
        turnPageFilterInfo.setCountTotalNum(100);
        turnPageFilterInfo.setCurCount(0);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
//                .httpUrl("https://www.dongchedi.com/feed")
                .httpUrl("https://www.dongchedi.com/auto/series/96")
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))//最大回溯时间范围
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")))
                .turnPageFilterInfo(turnPageFilterInfo)
                .needWashed(false)
                .proxy(proxy)
                .build();
        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
        return requestRecord;
    }




}
